import copy
import json
import types

from re_common.baselibrary.tools.all_requests.aiohttp_request import AioHttpRequest
from re_common.baselibrary.tools.all_requests.mrequest import MRequest
from re_common.baselibrary.utils.core.mdeprecated import try_except2_async, try_except2, retry_func_async


from apps.allsubdb.proxies_control.models import PostProxy
from apps.crawler_platform.core_abs.downstepabs import DownStep, PareAdapter, deal_format, deal_load
from apps.crawler_platform.core_callback import *
from apps.crawler_platform.core_callback.err_callback import all_client_back_callback
from apps.crawler_platform.core_platform.core_g import RedisKey
from apps.crawler_platform.core_platform.core_nosql import NoSqlUtil
from apps.crawler_platform.core_platform.core_sql import CoreSqlMixin
from apps.crawler_platform.core_platform.g_model import InputPlatformModel, TaskAllModel, \
    TaskInfoModel, TaskInfoSaveSettingModel, RedisAllTaskModel
from settings import get_settings

STATICA_HACK = False
# globals()['kcah_acitats'[::-1].upper()] = False
if STATICA_HACK:  # pragma: no cover
    null()


class BaseDownStep(DownStep):

    def __init__(self):
        super().__init__()
        self.pare_adapter = PareAdapter()

    async def redis_callback(self, tmpdict: RedisAllTaskModel) -> RedisAllTaskModel:
        name = "_".join([self.pm.task_name, self.pm.task_tag, "redis_callback"])
        if name in globals():
            callback = eval(name)
            return await callback(tmpdict)
        else:
            return tmpdict

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def get_redis(self):
        """
        获取redis的缓存
        """
        # 获取redis缓存的配置
        keys = self.pm.task_name + "_" + self.pm.task_tag
        task_set = await self.pm.redis.hget(RedisKey.taskinfo_save_setting, keys)
        if not task_set:
            raise Exception("没有配置" + keys + "在" + RedisKey.taskinfo_save_setting + "表。")
        task_save_setting = TaskInfoSaveSettingModel.parse_raw(task_set)
        tmpdict = {}
        keys += "_"
        async for name, val in self.pm.redis.ihscan(RedisKey.taskinfo, match='{}*'.format(keys)):
            tmp_key = name.replace(keys, "")
            tmpdict[tmp_key] = TaskAllModel(
                task_set=task_save_setting,
                task_info=TaskInfoModel.parse_raw(val)
            )
        self.redis_model = RedisAllTaskModel(parse_dict=tmpdict)
        self.redis_model = await self.redis_callback(self.redis_model)

        return True, ""

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    @retry_func_async(retry_times=3, sleep_time=1)
    async def get_proxy(self, task_name, task_tag):
        """
        获取代理接口
        """
        data = InputPlatformModel(
            data=PostProxy(task_name=task_name,
                           task_tag=task_tag
                           )).json()
        url = get_settings().PROXY_URL
        rrq = AioHttpRequest()
        rrq.set_url(url) \
            .set_timeout(30) \
            .set_data(data) \
            .set_middler_list(
            [rrq.is_null_html_middlerwares, rrq.status_code_middlerwares])
        bools, dicts = await rrq.run(MRequest.POST)
        from app_main import request_ctx_var
        request_ctx_var.get().state.proxy_html = rrq.html
        if bools:
            self.pm.proxy = json.loads(rrq.html)["data"]
            if self.pm.proxy == "":
                bools = False
        if not bools:
            self.add_process_info(self.step_info)
        return bools, dicts

    @retry_func_async(retry_times=3, sleep_time=1)
    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def down(self, keys):
        """
        基础下载逻辑
        :param keys:
        :return:
        """
        self.step_info = f"获取代理1,key is {keys};"
        bools, err_info = await self.get_proxy(self.pm.task_name, self.pm.task_tag)
        if not bools:
            return bools, err_info
        self.step_info = f"下载网页,key is {keys};"
        bools, err_info = await self.downler()
        return bools, err_info

    def func_add(self):
        """
        rrq的方法添加器,通过callbakck的_m_list后缀添加
        :return:
        """
        # 添加验证器
        name = "_".join([self.pm.task_name, self.pm.task_tag, "m", "list"])
        if name in globals():
            lists_m = eval(name)
            for item in lists_m:
                if item in self.pm.funcs:
                    func_m = types.MethodType(eval(item), self.rrq)
                    self.rrq.__dict__.update({item: func_m})

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def downler(self):
        """
        具体的下载逻辑,核心的下载器
        """
        self.rrq = AioHttpRequest()
        # 添加验证器
        self.func_add()
        Lists_middler = []
        self.rrq.set_url(self.pm.url) \
            .set_timeout(self.pm.time_out) \
            .set_header(self.pm.this_header)
        if self.pm.proxy:
            self.rrq.set_proxy(self.pm.proxy)

        if self.pm.resp_middler:
            self.rrq.set_middler_para(self.pm.resp_middler)
            for k, v in self.pm.resp_middler.items():
                Lists_middler.append(eval("self.rrq." + k))

        if self.pm.marks:
            self.rrq.set_marks(self.pm.marks)
            Lists_middler.append(self.rrq.marks_middlerwares)
        self.rrq.set_middler_list(Lists_middler)
        if self.pm.params:
            self.rrq.set_params(self.pm.params)
        if self.pm.req_type == "POST":
            if self.pm.post_data:
                self.rrq.set_data(self.pm.post_data)
            bools, dicts = await self.rrq.run(MRequest.POST)
        elif self.pm.req_type == "GET":
            bools, dicts = await self.rrq.run(MRequest.GET)
        else:
            raise Exception("req_type 错误;" + self.pm.req_type)
        return bools, dicts

    @try_except2()
    def para_redis(self, keys):
        """
        解析redis的缓存得到各种变量
        """
        # redis_model 中存储着所有key的配置
        self.pm.redis_model = self.redis_model
        self.pm.one_para = self.redis_model.parse_dict[keys]
        self.pm.task_tag_next = self.pm.one_para.task_info.task_tag_next
        self.pm.url = self.pm.one_para.task_info.url
        self.pm.url = deal_format(self.pm.url, self.pm.sql_model.get_dict())
        self.pm.req_type = self.pm.one_para.task_info.req_type
        self.pm.this_header = deal_load(self.pm.one_para.task_info.headers)
        for k, v in self.pm.this_header.items():
            v = deal_format(v, self.pm.sql_model.get_dict())
            self.pm.this_header[k] = v
        marks_t = deal_format(self.pm.one_para.task_info.marks, self.pm.sql_model.get_dict())
        self.pm.set_marks(marks_t)
        self.pm.resp_middler = deal_load(self.pm.one_para.task_info.resp_middler)
        self.pm.time_out = self.pm.one_para.task_info.timeout
        self.pm.validate_rules = ""
        if len(self.pm.one_para.task_info.validate_rules.strip()) > 0:
            self.pm.validate_rules = deal_load(self.pm.one_para.task_info.validate_rules)
        self.pm.order_num = self.pm.one_para.task_info.order_num
        self.pm.groups = self.pm.one_para.task_info.groups
        self.pm.funcs = self.pm.one_para.task_info.funcs
        self.pm.turn_page = self.pm.one_para.task_info.turn_page
        self.pm.state_key = self.pm.one_para.task_info.state_key
        self.pm.post_data_is_json = self.pm.one_para.task_info.post_data_is_json
        self.pm.set_post_data()
        self.pm.set_params()
        return True, ""

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def para_next(self):
        """
        解析得出下一级需要的数据
        """
        bools, dicts = await self.pare_adapter.pare_htmlmodel(self.pm.down_model, self.redis_model)
        if self.pare_adapter.para_dicts is not None:
            self.para_dicts = self.pare_adapter.para_dicts
        return bools, dicts

    @retry_func_async(retry_times=3, sleep_time=2)
    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def save_nosql_html(self):
        """
        存储到nosql数据库
        """
        # 期刊列表级默认保存历史
        self.pm.set_para()
        return await NoSqlUtil.save_nosql_html(self.pm)

    @retry_func_async(retry_times=3, sleep_time=2)
    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def read_nosql_html(self):
        """
        存储到nosql数据库
        """
        # 期刊列表级默认保存历史
        return await NoSqlUtil.read_nosql_html(self.pm)

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def err_collect(self, err_info, pm, batch, tag):
        if isinstance(err_info, dict):
            v = {"err_msg": str(err_info)}
            await CoreSqlMixin.insert_errinfo(pm.task_name, pm.task_tag, pm.journal_rawid, batch, v["err_msg"],
                                              tag)
            err_info["journal_rawid"] = pm.journal_rawid
            err_info["err_msg"] = v["err_msg"]
            return True, err_info

        else:
            return False, err_info

    @try_except2_async(callback=all_client_back_callback, is_print=True)
    async def save_sql(self, table, type="", code=""):
        # 中途失败的状态码更新到sql
        if type == "save_state":
            where = self.pm.get_pkm()
            vv = {self.pm.one_para.task_set.fianl_stat: code}
            update_no_placeholder = self.pm.get_update_no_placeholder()
            bools, dicts = await CoreSqlMixin.update_sql(table, vv, update_no_placeholder, where)
            return bools, dicts
        if type == "set_failcount":
            update_no_placeholder = {}
            for keys in code.split(";"):
                state_key_dict = self.pm.get_state_key(type="dict")
                if keys in state_key_dict.keys():
                    fail_key = state_key_dict[keys]
                    if fail_key in self.pm.sql_fail_key_set:
                        continue
                    else:
                        update_no_placeholder[fail_key] = fail_key + "+1"
                        # 加入set缓存，防止对一个键多次加1
                        self.pm.sql_fail_key_set.add(fail_key)
            self.pm.set_update_no_placeholder(update_no_placeholder, is_update=True)
        else:
            raise Exception("没有type的匹配项")
